package com.feng.mainPack;

import java.io.IOException;
import java.util.Iterator;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

public class GetUrls extends Thread {
	
	String sUrl;
	public GetUrls(String sUrl) {
		this.sUrl = sUrl;
	}

	@Override
	public void run() {
		
		//防止博客数目太少而PageUrls里面为空
		get(sUrl);

		while (QueueList.PageUrls.size() > 0) {

			// url很少时才执行

			String pUrl = QueueList.PageUrls.poll();
			
			//防止另外的线程调用此链接
			QueueList.VisitiedPageUrls.offer(pUrl);

			get(pUrl);


			System.out.println("剩余ArticleUrl" + QueueList.PageUrls.size());

		}

	}

	public void get(String url) {

		System.out.println("正在从" + url + "获取文章链接");

		GetContent gc = new GetContent();

		Document doc = null;
		try {
			doc = Jsoup.connect(url).timeout(30000).get();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		Iterator<Element> ie = doc.getElementsByClass("postTitle").iterator();

		// 获取postTitle目录下的所有元素
		while (ie.hasNext()) {
			Iterator<Element> ie2 = ie.next().getAllElements().iterator();

			while (ie2.hasNext()) {
				Element e1 = ie2.next();

				// 获取a标签
				if (e1.tagName() == "a") {

					// System.out.print(e1.text()+":");
					// System.out.println();

					String ur = e1.attr("href");

					if (ur == null || QueueList.VisitedArticleUrls.contains(ur) || QueueList.ArticleUrls.contains(ur)) {
						continue;
					}else{
						
						System.out.println("获取的链接:"+ur);
						
						QueueList.ArticleUrls.offer(ur);
						
					}


				}

			}

		}

		System.out.println("获取成功");

	}

}
